/*
 * Copyright 2024 KylinSoft Co., Ltd.
 *
 * This program is free software: you can redistribute it and/or modify it under
 * the terms of the GNU General Public License as published by the Free Software
 * Foundation, either version 3 of the License, or (at your option) any later
 * version.
 *
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along with
 * this program. If not, see <https://www.gnu.org/licenses/>.
 */

#include "autotokenizer.h"
#include "utils/pythonutil.h"

#include <iostream>

AutoTokenizer::AutoTokenizer(const std::string& modelFolderPath)
{
    if (!Py_IsInitialized())
        Py_Initialize();

    PythonThreadLocker pythonThreadLocker;

    //python c api函数  请求python解释器执行字符串，作用是导入python的标准库模块sys
    PyRun_SimpleString("import sys");
    //调用sys的path.append函数进行字符串拼接
    std::string command = "sys.path.append('" + std::string(DATA_MANAGEMENT_PYTHON_PATH) + "')";
    PyRun_SimpleString(command.c_str());

    PyObject* module = PyImport_ImportModule("autotokenizer");
    if (!module) {
        std::cerr << "Import tokenizer error" << std::endl;
        PyErr_Print();
        return;
    }

    PyObject* tokenizerClass = PyObject_GetAttrString(module, "Tokenizer");
    if (!tokenizerClass) {
        std::cerr << "Get tokenizer class error" << std::endl;
        PyErr_Print();
        Py_XDECREF(module);
        return;
    }

    PyObject* args = Py_BuildValue("(s)", modelFolderPath.c_str());
    if (!args) {
        std::cerr << "Build string args with " << modelFolderPath << " error" << std::endl;
        PyErr_Print();
        Py_XDECREF(module);
        Py_XDECREF(tokenizerClass);
        return;
    }

    tokenizer_ = PyObject_CallObject(tokenizerClass, args);
    if (!tokenizer_) {
        std::cerr << "Call init_tokenizer error" << std::endl;
        PyErr_Print();
    }

    Py_XDECREF(module);
    Py_XDECREF(tokenizerClass);
    Py_XDECREF(args);
}

AutoTokenizer::~AutoTokenizer()
{
    PythonThreadLocker pythonThreadLocker;

    if (tokenizer_ != nullptr)
        Py_XDECREF(tokenizer_);
}

Token AutoTokenizer::tokenize(const std::string &text)
{
    PythonThreadLocker pythonThreadLocker;
    std::map<std::string, std::vector<int64_t>> result;

    if (!Py_IsInitialized()) {
        std::cerr << "Don't init python enveriment, can't tokenize" << std::endl;
        return result;
    }

    if (tokenizer_ == nullptr) {
        std::cerr << "Tokenizer don't be init, can't tokenize" << std::endl;
        return result;
    }

    PyObject* tokenizeFunc = PyObject_GetAttrString(tokenizer_, "tokenize_text");
    if (!tokenizeFunc || !PyCallable_Check(tokenizeFunc)) {
        std::cout << "Don't found function tokenize_text" << std::endl;

        Py_XDECREF(tokenizeFunc);
        return result;
    }

    PyObject* args = Py_BuildValue("(s)", text.c_str());
    PyObject* pyResult = PyObject_CallObject(tokenizeFunc, args);

    if (!pyResult) {
        Py_XDECREF(args);
        Py_XDECREF(tokenizeFunc);
        return result;
    }
    if (PyDict_Check(pyResult)) {
        result = pythonutil::convertPythonDictToStdMap(pyResult);
    }

    Py_XDECREF(args);
    Py_XDECREF(pyResult);
    Py_XDECREF(tokenizeFunc);

    return result;
}
